/**
 * 
 */
package com.crawl;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.htmlparser.Parser;
import org.htmlparser.filters.AndFilter;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;

import com.corpus.KeyWordSet;
import com.link.LinkDB;

/**
 * @author LvSaiHui {gurnfe@163.com}
 * @version 1.0 <br>
 *          Copyright (C), 2007-2008, ZJUT <br>
 *          This program is protected by copyright laws. <br>
 *          Program Name: MetaSearch.java <br>
 *          Date: Dec 30, 2008 <br>
 *          Description:
 */
public class MetaSearch {
	public Log log = LogFactory.getLog(MetaSearch.class);

	public void searchByWordList(String filename) throws IOException {
		try {
			String buffer;
			FileWriter resultFile = null;
			PrintWriter myFile = null;

			String dstfile = filename + "_dsturl.txt";
			File writefile = new File(dstfile);
			if (!writefile.exists()) {
				writefile.createNewFile();
			}
			resultFile = new FileWriter(writefile);
			myFile = new PrintWriter(resultFile);
			// BufferedReader reader = new BufferedReader(new
			// FileReader(filename));
			InputStreamReader read = new InputStreamReader(new FileInputStream(
					filename), "UTF-8");
			BufferedReader reader = new BufferedReader(read);
			while ((buffer = reader.readLine()) != null
					&& buffer.trim().length() > 0) {
				String url = "http://www.baidu.com/s?lm=0&si=&rn=10&ie=gb2312&ct=0&wd="
						+ URLEncoder.encode(buffer, "gb2312")
						+ "&pn=0&ver=0&cl=3";
				// String url="http://www.baidu.com/s?wd="+buffer;
				System.out.println(buffer);
				System.out.println(url);
				getBaiduUrls(url, "GB2312", myFile);
			}

			if (myFile != null)
				myFile.close();
			if (resultFile != null)
				resultFile.close();

		} catch (ParserException e) {
			e.printStackTrace();
		}

	}

	public String[] getBaiduUrls(String url, String pageEncoding,
			PrintWriter writer) throws ParserException {
		NodeList nodeList = null;
		String[] ret = null;
		try {
			Parser parser = new Parser(url);
			parser.setEncoding(pageEncoding);
			nodeList = parser.parse(new AndFilter(new HasAttributeFilter(
					"target"), new HasAttributeFilter("onmousedown")));
			ret = new String[nodeList.size()];

		} catch (ParserException e) {
			e.printStackTrace();
		}
		if (nodeList != null && nodeList.size() > 0) {
			for (int i = 0; i < nodeList.size(); i++) {
				String urlLink = ((LinkTag) nodeList.elementAt(i))
						.extractLink();
				ret[i] = urlLink;
				String LinkName = ((LinkTag) nodeList.elementAt(i))
						.getLinkText();

				if (urlLink.indexOf("bnu") == 0 || urlLink.indexOf("http") == 0)
					System.out.println("****The result : " + i);
				System.out.println("The title : " + LinkName);
				System.out.println("The link : " + urlLink);
				writer.println(urlLink);
			}
		}
		return ret;
	}

	public static List<String> getBaiduUrls(String searchWord)
			throws UnsupportedEncodingException {

		return getBaiduUrls(searchWord, 50);

	}

	public static List<String> getBaiduUrls(String searchWord,
			int defautRecordCount) throws UnsupportedEncodingException {
		int pageNum = 0;

		NodeList nodeList = null;
		List<String> ret = new ArrayList<String>();
		for (pageNum = 0; pageNum < defautRecordCount / 10; pageNum++) {
			String url = "http://www.baidu.com/s?lm=0&si=&rn=10&ie=gb2312&ct=0&wd="
					+ URLEncoder.encode(searchWord, "gb2312")
					+ "&pn="
					+ pageNum * 10 + "&ver=0&cl=3";
			try {
				Parser parser = new Parser(url);
				parser.setEncoding("GB2312");
				nodeList = parser.parse(new AndFilter(new HasAttributeFilter(
						"target"), new HasAttributeFilter("onmousedown")));

			} catch (ParserException e) {
				e.printStackTrace();
			}

			if (nodeList != null && nodeList.size() > 0) {
				// int startIndex=pageNum > 0 ? 0 : 1;
				for (int i = 0; i < nodeList.size(); i++) {
					String urlLink = ((LinkTag) nodeList.elementAt(i))
							.extractLink();
					ret.add(urlLink);
					System.out.println("The link : " + urlLink);
					/*
					 * String LinkName = ((LinkTag) nodeList.elementAt(i))
					 * .getLinkText();
					 * 
					 * if (urlLink.indexOf("bnu") == 0 ||
					 * urlLink.indexOf("http") == 0)
					 * System.out.println("****The result : " + i);
					 * System.out.println("The title : " + LinkName);
					 * System.out.println("The link : " + urlLink);
					 */
				}
			} else {
				break;
			}
		}
		// 有时候含有对中文的百度词典查询，所以要踢出这个网址，就像如果我搜索“信用”，会含有
		// http://www.baidu.com/s?lm=0&si=&rn=10&ie=gb2312&ct=1048576&wd=%D0%C5%D3%C3&tn=baidu
		/*Iterator<String> it = ret.iterator();
		String targetUrl = null;
		while (it.hasNext()) {
			String url = (String) it.next();
			if (url.indexOf("http://www.baidu.com/") >= 0) {
				targetUrl = url;
			}
		}
		ret.remove(targetUrl);
*/
		if(ret.get(0).indexOf("http://www.baidu.com/")>=0){
			ret.remove(0);
		}
		return ret;

	}

	public String[] getGoogleUrls() {
		String[] ret = null;
		return ret;
	}

	public static void main(String[] args) throws IOException {
		// MetaSearch metaSearch = new MetaSearch() //
		// metaSearch.searchByWordList("D:\\eclipse_workplace\\crawler\\src\\wordlist.txt");
		/*
		 * String text="信用";
		 * System.out.println(URLEncoder.encode(text,"gb2312"));
		 */
		List<String> set = MetaSearch.getBaiduUrls("信用", 32);
		Iterator it = set.iterator();
		int j = 0;
		while (it.hasNext()) {
			System.out.println(j + "  : " + it.next());
			j++;
		}

	}

	public static String getMutationUrlBasedCorpusSearch()
			throws UnsupportedEncodingException {

		Iterator it = KeyWordSet.Theme.entrySet().iterator();
		List urlList = null;
		String returnUrl = null;
		while (it.hasNext()) {
			Map.Entry entry = (Map.Entry) it.next();
			String keyword = (String) entry.getKey();
			double value = (Double) entry.getValue();
			Random rd = new Random();
			if (value > 5 && rd.nextInt(100) < 20) {
				urlList = getBaiduUrls(keyword);
			}

		}
		if (!urlList.isEmpty()) {
			Iterator iterator = urlList.iterator();
			while (iterator.hasNext()) {
				String temp = (String) iterator.next();
				if (!LinkDB.getUnVisitedUrl().contains(temp)
						&& !LinkDB.getVisitedUrl().contains(temp)) {
					returnUrl = temp;
					break;
				}

			}

		}

		return returnUrl;

	}

}
